// =============================================================
// CIndexInterpreter interface
// ---------------------------------------
// Author : Woo-Hyun Kim
// Email : woorung@icu.ac.kr or woorung@nhncorp.com
// Date : Aug. 8, 2006
//----------------------------------------------------------------------
#include "CInfoMap.h"
#include "CIndexInterpreter.h"
#include "CShellProcess.h"
//----------------------------------------------------------------------
CIndexInterpreter::CIndexInterpreter(string &name) : CInterpreter(name)
{
}
//----------------------------------------------------------------------
CIndexInterpreter::~CIndexInterpreter()
{
}
//----------------------------------------------------------------------
// bind the requested calls to real methods
//----------------------------------------------------------------------
void CIndexInterpreter::run() {
	if(call_name=="sp_initialize_environment") initializeEnvironment(call_name, params);
	else if(call_name=="sp_fetch_by_rsync") fetchByRSYNC(call_name, params);
	else if(call_name=="sp_dispatch_by_rsync") dispatchByRSYNC(call_name, params);
	else if(call_name=="sp_makeup_common_index_block") makeupCommonIndexBlock(call_name, params);
	else if(call_name=="sp_makeup_index_block") makeupIndexBlock(call_name, params);
	else if(call_name=="sp_index") index(call_name, params);
	else if(call_name=="sp_makeup_nxp") makeupNXP(call_name, params);
	else if(call_name=="sp_nxp") nxp(call_name, params);
	else {
		cout << "///////////////////////////////////////////////////////////" << endl;
		cout << "[error] : Not found such an invoked method : " << call_name << endl;
		cout << "///////////////////////////////////////////////////////////" << endl;
		exit(-1);
	}
}
//----------------------------------------------------------------------
// common functions
//----------------------------------------------------------------------
string &CIndexInterpreter::getExportedLibsCommand(map<string,string> &parameters, string &command)
{
	// build commands to export shared libraries
	string export_libs = "LD_LIBRARY_PATH=";
	export_libs += (parameters["naver_linguist_lib_path"] + ":");
	export_libs += (parameters["naver_sigmasearch_lib_path"] + ":");
	export_libs += (parameters["naver_nxp_lib_path"] + ":");
	export_libs += "${LD_LIBRARY_PATH};";
	export_libs += "export LD_LIBRARY_PATH;";
	
	command = export_libs;
	return command;
}
//----------------------------------------------------------------------
string &CIndexInterpreter::executeShellCommand(string &command)
{
    char buf[1024];
    memset((char *)buf, '\0', sizeof(buf));

    FILE *p = popen(command.c_str(), "r");
    if(!p) {
        cout << "pipe open error : " << command << endl;
    }
	else cout << "[execute] " << command << endl;

    // to contain the result of the command execution
    string tmp = "";
    while(fgets(buf, sizeof(buf)-1, p)!=0) {
        tmp += buf;
        memset((char *)buf, '\0', sizeof(buf));
    }

    pclose(p);

    string result;
    int len = tmp.length();
    if(tmp[len-1]=='\n') result = tmp.substr(0,tmp.length()-1);
    else result = tmp;

    command = result;

    return command;
}
//----------------------------------------------------------------------
void CIndexInterpreter::executeSigmaDocumentLocation(map<string,string> &parameters)
{
	char tmp_path[1024];
	sprintf(tmp_path,"%s/%s/%s/",parameters["sp_idx"].c_str(), parameters["sp_today"].c_str(), parameters["uspec_index_group_name"].c_str());

	string index_home = tmp_path;
	string data_coll = index_home + "data_coll";
	string indexed_coll = index_home + "indexed_coll";
	string uspec_file_path = indexed_coll + "/SPEC.USR";

	string docs_today = parameters["sp_docs"] + "/" + parameters["sp_today"] + "/";
	string file_to_index = docs_today + parameters["uspec_file_to_index"];

	// build sigmadloc command
	string command = parameters["naver_sigmasearch_lib_path"];
	command += "/bin/sigmadloc ";
	command += ("-U " + uspec_file_path + " ");
	command += "-b ";
	command += ("-s \"" + parameters["iinfo_begin_tag"] + "\" ");
	command += file_to_index;

	cout << "[execute] : " << command << endl;

	system(command.c_str());
}
//----------------------------------------------------------------------
void CIndexInterpreter::executeSigmaIndex(map<string,string> &parameters)
{
	char tmp_path[1024];
	sprintf(tmp_path,"%s/%s/%s/",parameters["sp_idx"].c_str(), parameters["sp_today"].c_str(), parameters["uspec_index_group_name"].c_str());

	string index_home = tmp_path;
	string data_coll = index_home + "data_coll";
	string indexed_coll = index_home + "indexed_coll";
	string uspec_file_path = indexed_coll + "/SPEC.USR";
	string docs_today = parameters["sp_docs"] + "/" + parameters["sp_today"] + "/";
	string file_to_index = docs_today + parameters["uspec_file_to_index"];

	// get index names from parameters[iinfo_index_name]
	string delim = " ";
	list<string> indexes;
	CSmartString *sm = new CSmartString(parameters["iinfo_index_name"],delim);
	sm->CSplit(indexes);
	delete sm;

	// build basic sigmadloc command                  
	string command = parameters["naver_sigmasearch_lib_path"];
	command += "/bin/sigmaindex ";
	command += ("-U " + uspec_file_path + " ");

	// build a complete index command with options
	char options[1024];
	string index_cmd;
	list<string>::iterator index = indexes.begin();
	while(index!=indexes.end()) {
		// initialize
		sprintf(options, "-i %s -s %s -n %s", (*index).c_str(), parameters["index_size_of_index_volume"].c_str(), parameters["index_num_of_processes"].c_str());
		index_cmd = command;
		index_cmd += options;
		cout << "[index 0] : " << index_cmd << endl;
		system(index_cmd.c_str());
		cout << endl;

		// create tmp directory to index
		string tmp_dir_to_index = indexed_coll + "/" + *index + "_tmp";
		generateDirectory(parameters["sp_home"], tmp_dir_to_index);

		// 1st step
		sprintf(options,"-1 %s -t %s -n %s -M %s", (*index).c_str(), tmp_dir_to_index.c_str(), parameters["index_num_of_processes"].c_str(), parameters["index_size_of_memory_per_process"].c_str());
		index_cmd = command;
		index_cmd += options;
		cout << "[index 1] : " << index_cmd << endl;
		system(index_cmd.c_str());
		cout << endl;

		// 2nd step
		sprintf(options,"-2 %s -t %s -p %s", (*index).c_str(), tmp_dir_to_index.c_str(), parameters["index_size_of_memory_to_merge"].c_str());
		index_cmd = command;
		index_cmd += options;
		cout << "[index 2] : " << index_cmd << endl;
		system(index_cmd.c_str());
		cout << endl;

		// remove tmp directory
		string rmdir_cmd = "rm -rf ";
		rmdir_cmd += tmp_dir_to_index;
		system(rmdir_cmd.c_str());

		index++;
	}
}
//----------------------------------------------------------------------
void CIndexInterpreter::executeSigmaFilter(map<string,string> &parameters)
{
    char tmp_path[1024];
    sprintf(tmp_path,"%s/%s/%s/",parameters["sp_idx"].c_str(), parameters["sp_today"].c_str(), parameters["uspec_index_group_name"].c_str());

    string index_home = tmp_path;
    string data_coll = index_home + "data_coll";
    string indexed_coll = index_home + "indexed_coll";
    string uspec_file_path = indexed_coll + "/SPEC.USR";
    string docs_today = parameters["sp_docs"] + "/" + parameters["sp_today"] + "/";
    string file_to_index = docs_today + parameters["uspec_file_to_index"];

    // get index names from parameters[iinfo_filter]
    string delim = " ";
    list<string> filters;
    CSmartString *sm = new CSmartString(parameters["iinfo_filter"],delim);
    sm->CSplit(filters);
    delete sm;

    // build basic sigmafilter command
    string command = parameters["naver_sigmasearch_lib_path"];
    command += "/bin/sigmafilter ";
    command += ("-U " + uspec_file_path + " ");

	// build a complete filter command with options
	char options[1024];
	string filter_cmd;
	list<string>::iterator filter = filters.begin();
	while(filter!=filters.end()) {
		if(*filter=="intflt") {
			filter_cmd = command;
			filter_cmd += "intflt";
			cout << "[intflt] : " << filter_cmd << endl;
			system(filter_cmd.c_str());
		}
		else if(*filter=="floatflt") {
			filter_cmd = command;
			filter_cmd += "floatflt";
			cout << "[floatflt] : " << filter_cmd << endl;
			system(filter_cmd.c_str());
		}
		else if(*filter=="vintflt") {
			filter_cmd = command;
			filter_cmd += "vintflt";
			cout << "[vintflt] : " << filter_cmd << endl;
			system(filter_cmd.c_str());
		}
		else if(*filter=="charflt") {
			filter_cmd = command;
			filter_cmd += "charflt";         
			cout << "[charflt] : " << filter_cmd << endl;
			system(filter_cmd.c_str());                              
		}
		else if(*filter=="strflt") {
			filter_cmd = command;
			filter_cmd += "strflt";         
			cout << "[strflt] : " << filter_cmd << endl;
			system(filter_cmd.c_str());                              
		}

		filter++;
	}
}
//----------------------------------------------------------------------
void CIndexInterpreter::startNXPFromConfFiles(map<string,string> &parameters)
{
	// build basic nxpserver/broker command
	string command = parameters["naver_nxp_lib_path"];
	command += "/bin/nxpserver ";
	command += "-l " + parameters["sp_log"] + " ";
	command += "-f ";

	string delim = ",:; \t";

	list<string> nxphosts;
	CSmartString *sm = new CSmartString(parameters["nxp_host"], delim);
	sm->CSplit(nxphosts);
	delete sm;

	vector<string> files;
	int level = 1;
	CFileManager *fileman = new CFileManager();
	fileman->StartWalking((char *)parameters["sp_conf"].c_str(), level);
	files = fileman->getFiles(files);
	delete fileman;

	// build commands to export shared libraries
	string exported_libs_cmd;
	exported_libs_cmd = getExportedLibsCommand(parameters, exported_libs_cmd);

	// local execution ----------------------------------
	if(parameters["nxp_host"]=="") {
		string nxp_cmd;
		for(int i=0;i<files.size();i++) {
			string file = files[i];
			string onlyfile;

			int found = file.rfind("_nxp.conf");
			if(found<0||found>=file.length()) continue;
			else {
				nxp_cmd = exported_libs_cmd + command;
				nxp_cmd += file;

				int start = file.rfind("/");
				int stop = file.rfind(".");
				if(stop<0||stop>=file.length()) stop = file.length();
				if(start>stop) stop = file.length();

				if(start<0||start>=file.length()) onlyfile = file.substr(0,stop);
				else onlyfile = file.substr(start+1,stop-(start+1));

				string back_log = " >> " + parameters["sp_log"] + "/";
				back_log += onlyfile + "." + parameters["sp_today"] + ".log";
				back_log += " 2>&1 && sleep 2 2>&-";

				cout << "[nxpserver] : " << nxp_cmd << endl;
				system(nxp_cmd.c_str());
				cout << endl;
			}
		}
	}
	// remote execution -----------------------------------
	else {
		list<string>::iterator nxphost = nxphosts.begin();
		while(nxphost!=nxphosts.end()) {
			string nxp_cmd;
			for(int i=0;i<files.size();i++) {
				string file = files[i];
				string onlyfile;

				int found = file.rfind("_nxp.conf");
				if(found<0||found>=file.length()) continue;
				else {
					nxp_cmd = exported_libs_cmd + command;
					nxp_cmd += file;

					int start = file.rfind("/");
					int stop = file.rfind(".");
					if(stop<0||stop>=file.length()) stop = file.length();
					if(start>stop) stop = file.length();

					if(start<0||start>=file.length()) onlyfile = file.substr(0,stop);
					else onlyfile = file.substr(start+1, stop-(start+1));

					string back_log = " >> " + parameters["sp_log"] + "/";
					back_log += onlyfile + "." + parameters["sp_today"] + ".log";
					back_log += " 2>&1 && sleep 2 2>&-";

					string remote_nxp_cmd = "ssh " + *nxphost;
					remote_nxp_cmd += " \"";
					remote_nxp_cmd += nxp_cmd;
					remote_nxp_cmd += back_log;
					remote_nxp_cmd += "\"";
					cout << "[nxpserver] : " << remote_nxp_cmd << endl;
					system(remote_nxp_cmd.c_str());
					cout << endl;
				}
			}

			nxphost++;
		}
	}
}
//----------------------------------------------------------------------
void CIndexInterpreter::stopNXPFromConfFiles(map<string,string> &parameters)
{
	string delim = ",:; \t";

	list<string> nxphosts;
	CSmartString *sm = new CSmartString(parameters["nxp_host"], delim);
	sm->CSplit(nxphosts);
	delete sm;

	vector<string> files;
	int level = 1;
	CFileManager *fileman = new CFileManager();
	fileman->StartWalking((char *)parameters["sp_conf"].c_str(), level);
	files = fileman->getFiles(files);
	delete fileman;

	// local execution -------------------------------------------
	if(parameters["nxp_host"]=="") {
		string pids_cmd;
		string success_pids = ""; 
		string failure_pids = ""; 
		for(int i=0;i<files.size();i++) {
			string file = files[i];

			int found = file.rfind("_nxp.conf");
			if(found<0||found>=file.length()) continue;
			else {
				pids_cmd = "ps -eo pid,cmd | grep ";
				pids_cmd += file;

				string pids = "";

				pids_cmd += " | grep -v grep | awk '{print $1}' | xargs";

				pids = executeShellCommand(pids_cmd);

				list<string> nxps;
				CSmartString *sm = new CSmartString(pids, delim);
				sm->CSplit(nxps);
				delete sm;

				string kill_cmd;
				list<string>::iterator nxp_pid = nxps.begin();
				while(nxp_pid!=nxps.end()) {
					int killing_pid = atoi((*nxp_pid).c_str());
					int count = 0;

					while(kill(killing_pid, SIGKILL)!=0) {
						if(count++>5) { // finally failed after trying 5 times
							failure_pids += (*nxp_pid + " ");
							break;
						}
						sleep(3);
					}

					if(count<=5) { // successfully killed
						success_pids += (*nxp_pid + " ");
					}

					nxp_pid++;
				}
			}
		}

		cout << "[nxpserver] killed all the nxpservers : " << success_pids << endl;
		if(failure_pids!="") cout << "[nxpserver] error : cannot kill the processes (" << failure_pids << ")" << endl;
	}

	// remote execution -------------------------------------------
	else {
		list<string>::iterator nxphost = nxphosts.begin();
		while(nxphost!=nxphosts.end()) {
			string pids_cmd;
			string success_pids = "";
			string failure_pids = "";
			for(int i=0;i<files.size();i++) {
				string file = files[i];

				int found = file.rfind("_nxp.conf");
				if(found<0||found>=file.length()) continue;
				else {
					pids_cmd = "ps -eo pid,cmd | grep ";
					pids_cmd += file;

					string pids = "";

					string remote_pids_cmd = "ssh " + *nxphost;
					remote_pids_cmd += " \"" + pids_cmd;
					// attention : '$' character must be escaped
					remote_pids_cmd += " | grep -v grep | awk '{print \\$1}' | xargs";
					remote_pids_cmd += "\"";

					pids = executeShellCommand(remote_pids_cmd);

					list<string> nxps;
					CSmartString *sm = new CSmartString(pids, delim);
					sm->CSplit(nxps);
					delete sm;

					string kill_cmd;
					list<string>::iterator nxp_pid = nxps.begin();
					while(nxp_pid!=nxps.end()) {
						int killing_pid = atoi((*nxp_pid).c_str());
						int count = 0;

						string remote_kill_cmd = "ssh " + *nxphost;
						remote_kill_cmd += " \"kill -9 ";
						remote_kill_cmd += *nxp_pid;
						remote_kill_cmd += "\"";

						do {
							if(count++>5) { // finally failed after trying 5 times
								failure_pids += (*nxp_pid + " ");
								break;
							}
							string kill_results = executeShellCommand(remote_kill_cmd);
							int success = kill_results.find("No such process");
							if(success<0||success>=kill_results.length()) break;
							sleep(3);
						} while(1);

						if(count<=5) { // successfully killed
							success_pids += (*nxp_pid + " ");
						}

						nxp_pid++;
					}
				}
			}

			nxphost++;
				
			cout << "[nxpserver] killed all the nxpservers : " << success_pids << endl;
			if(failure_pids!="") cout << "[nxpserver] error : cannot kill the processes (" << failure_pids << ")" << endl;

		}
	}
}
//----------------------------------------------------------------------
void CIndexInterpreter::startNXP(map<string,string> &parameters)
{
	if(parameters["nxp_server_name"]=="") {
		startNXPFromConfFiles(parameters);
	}
	else {
		// build basic nxpserver/broker command
		string command = parameters["naver_nxp_lib_path"];
		command += "/bin/nxpserver ";
		command += ("-l " + parameters["sp_log"] + " ");
		command += ("-f " + parameters["sp_conf"] + "/");

		string delim = ",:; \t";

		list<string> nxphosts;
		CSmartString *sm1 = new CSmartString(parameters["nxp_host"], delim);
		sm1->CSplit(nxphosts);
		delete sm1;

		list<string> snames;
		CSmartString *sm2 = new CSmartString(parameters["nxp_server_name"], delim);
		sm2->CSplit(snames);
		delete sm2;

		string exported_libs_cmd;
		exported_libs_cmd = getExportedLibsCommand(parameters, exported_libs_cmd);

		// local execution ---------------------------
		if(parameters["nxp_host"]=="") {
			string nxp_cmd;
			list<string>::iterator sname = snames.begin();
			while(sname!=snames.end()) {
				nxp_cmd = (exported_libs_cmd + command);
				nxp_cmd += *sname;
				nxp_cmd += "_nxp.conf";

				string back_log = " >> " + parameters["sp_log"] + "/";
				back_log += *sname + "_nxp." + parameters["sp_today"] + ".log";
				back_log += " 2>&1 && sleep 2 2>&-";

				nxp_cmd += back_log;

				cout << "[nxpserver] : " << nxp_cmd << endl;
				system(nxp_cmd.c_str());
				cout << endl;

				sname++;
			}
		}
		// remote exeuction ----------------------------
		else {
			list<string>::iterator nxphost = nxphosts.begin();
			while(nxphost!=nxphosts.end()) {
				string nxp_cmd;
				list<string>::iterator sname = snames.begin();
				while(sname!=snames.end()) {
					nxp_cmd = (exported_libs_cmd + command);
					nxp_cmd += *sname;
					nxp_cmd += "_nxp.conf";

					string back_log = " >> " + parameters["sp_log"] + "/";
					back_log += *sname + "_nxp." + parameters["sp_today"] + ".log";
					back_log += " 2>&1 && sleep 2 2>&-";

					string remote_nxp_cmd = "ssh " + *nxphost;
					remote_nxp_cmd += " \"";
			   		remote_nxp_cmd += nxp_cmd;
					remote_nxp_cmd += back_log;
					remote_nxp_cmd += "\"";
					cout << "[nxpserver] : " << remote_nxp_cmd << endl;
					system(remote_nxp_cmd.c_str());
					cout << endl;

					sname++;
				}

				nxphost++;
			}
		}
	}
}
//----------------------------------------------------------------------
void CIndexInterpreter::stopNXP(map<string,string> &parameters)
{
	if(parameters["nxp_server_name"]=="") {
		stopNXPFromConfFiles(parameters);
	}
	else {
		string delim = ",:; \t";

		list<string> nxphosts;
		CSmartString *sm1 = new CSmartString(parameters["nxp_host"], delim);
		sm1->CSplit(nxphosts);
		delete sm1;

		list<string> snames;
		CSmartString *sm2 = new CSmartString(parameters["nxp_server_name"], delim);
		sm2->CSplit(snames);
		delete sm2;

		// local execution -------------------------------------
		if(parameters["nxp_host"]=="") {
			string pids_cmd;
			string success_pids = "";
			string failure_pids = "";

			list<string>::iterator sname = snames.begin();
			while(sname!=snames.end()) {
				string file = *sname + "_nxp.conf";

				pids_cmd = "ps -eo pid,cmd | grep ";
				pids_cmd += file;

				string pids = "";

				pids_cmd += " | grep -v grep | awk '{print $1}' | xargs";

				pids = executeShellCommand(pids_cmd);

				list<string> nxps;
				CSmartString *sm = new CSmartString(pids, delim);
				sm->CSplit(nxps);
				delete sm;

				string kill_cmd;
				list<string>::iterator nxp_pid = nxps.begin();
				while(nxp_pid!=nxps.end()) {
					int killing_pid = atoi((*nxp_pid).c_str());
					int count = 0;

					while(kill(killing_pid, SIGKILL)!=0) {
						cout << "trying to kill - count " << count+1 << endl;
						if(count++>5) { // finally failed after trying 5 times
							failure_pids += (*nxp_pid + " ");
							break;
						}
						sleep(3);
					}

					if(count<=5) { // successfully killed
						success_pids += (*nxp_pid + " ");
					}

					nxp_pid++;
				}

				sname++;
			}

			if(success_pids!="") cout << "[nxpserver] killed all the nxpservers : " << success_pids << endl;
			if(failure_pids!="") cout << "[nxpserver] error : cannot kill the processes (" << failure_pids << ")" << endl;
		}

		// remote execution -------------------------------------
		else {
			list<string>::iterator nxphost = nxphosts.begin();
			while(nxphost!=nxphosts.end()) {
				string pids_cmd;
				string success_pids = "";
				string failure_pids = "";

				list<string>::iterator sname = snames.begin();
				while(sname!=snames.end()) {
					string file = *sname + "_nxp.conf";

            		pids_cmd = "ps -eo pid,cmd | grep ";
            		pids_cmd += file;

					string pids = "";	

					string remote_pids_cmd = "ssh " + *nxphost;
					remote_pids_cmd += " \"" + pids_cmd;
					// attention : '$' character must be escaped
					remote_pids_cmd += " | grep -v grep | awk '{print \\$1}' | xargs";
					remote_pids_cmd += "\"";

					pids = executeShellCommand(remote_pids_cmd);

					list<string> nxps;
					CSmartString *sm = new CSmartString(pids, delim);
					sm->CSplit(nxps);
					delete sm;

					string kill_cmd;
					list<string>::iterator nxp_pid = nxps.begin();
					while(nxp_pid!=nxps.end()) {
						int killing_pid = atoi((*nxp_pid).c_str());
						int count = 0;

						string remote_kill_cmd = "ssh " + *nxphost;
						remote_kill_cmd += " \"kill -9 ";
						remote_kill_cmd += *nxp_pid;
						remote_kill_cmd += "\"";

						do {
							if(count++>5) { // finally failed after trying 5 times
								failure_pids += (*nxp_pid + " ");
								break;
							}
							string kill_results = executeShellCommand(remote_kill_cmd);
							int success = kill_results.find("No such process");
							if(success<0||success>=kill_results.length()) break;
							sleep(3);
						} while(1);

						if(count<=5) { // successfully killed
                    		success_pids += (*nxp_pid + " ");
                		}

                		nxp_pid++;
            		}

					sname++;
        		}

				nxphost++;

    			if(success_pids!="") cout << "[nxpserver] killed all the nxpservers : " << success_pids << endl;
    			if(failure_pids!="") cout << "[nxpserver] error : cannot kill the processes (" << failure_pids << ")" << endl;

			}
		}
	}
}
//----------------------------------------------------------------------
void CIndexInterpreter::inspectParameters(string &name, map<string,string> &parameters)
{
	cout << endl;
    cout << "[inspection] : " << name << endl;
	cout << "----------------------------------------------------------" << endl;

    map<string,string>::iterator param = parameters.begin();
    while(param!=parameters.end()) {
        cout << param->first << "=" << param->second << endl;

        param++;
    }
	cout << "----------------------------------------------------------" << endl << endl;
}
//----------------------------------------------------------------------
void CIndexInterpreter::generateDirectory(string &default_path, string &current_path)
{
    struct stat statbuf;
    struct dirent *dirp;
    DIR *dp;

    string curdir = default_path;
    string curpath = current_path;
    string delim = "/";

	int start = curpath.find(curdir);
	int len = curdir.size();
	if(start>=0&&start<len) curpath = curpath.substr(start+len);

    list<string> dirs;
    CSmartString *sm = new CSmartString(curpath, delim);
    sm->CSplit(dirs);
    delete sm;

    list<string>::iterator current = dirs.begin();
    while(current!=dirs.end()) {
        curdir += "/";
        curdir += *current;
        if((dp=opendir(curdir.c_str()))==NULL) {
            mkdir(curdir.c_str(),07777);
            cout << "[generate] : " << curdir << endl;
        }
        if(dp!=NULL) {
            //cout << "[already exist] : " << curdir << endl;
            closedir(dp);
        }

        current++;
    }
}
//----------------------------------------------------------------------
void CIndexInterpreter::removeDirectory(string &target_path)
{
	DIR *dp;
	struct dirent *ep;
	if((dp = opendir(target_path.c_str()))==NULL) {
		while(ep=readdir(dp)) {
			string d_name = ep->d_name;
			remove(d_name.c_str());
		}
	}
	if(dp!=NULL) {
		closedir(dp);
	}
}
//----------------------------------------------------------------------
// reflection APIs
//----------------------------------------------------------------------
//----------------------------------------------------------------------
// common methods for index
//----------------------------------------------------------------------
void CIndexInterpreter::initializeEnvironment(string &name, map<string,string> &parameters)
{
	string home = "";

	if(parameters["sp_home"]=="") {
		cout << "[sp_initialize_environment] error : @sp_home must be set" << endl;
		//return results;
	}

	generateDirectory(home, parameters["sp_home"]);

	home = parameters["sp_home"];

	generateDirectory(home, parameters["sp_docs"]);
	generateDirectory(home, parameters["sp_idx"]);
	generateDirectory(home, parameters["sp_var"]);
	generateDirectory(home, parameters["sp_sbin"]);
	generateDirectory(home, parameters["sp_bin"]);
	generateDirectory(home, parameters["sp_log"]);
	generateDirectory(home, parameters["sp_etc"]);
	generateDirectory(home, parameters["sp_conf"]);
	generateDirectory(home, parameters["sp_htdocs"]);
	generateDirectory(home, parameters["sp_src"]);

    //inspectParameters(name, parameters);

	//return results;
}
//----------------------------------------------------------------------
void CIndexInterpreter::deliverByRSYNC(string &name, map<string,string> &parameters)
{
	string delim = ",:; \t";
	list<string> from_files;
	CSmartString *from_sm =  new CSmartString(parameters["sp_from_file"],delim);
	from_sm->CSplit(from_files);
	delete from_sm;

	list<string> to_files;
	CSmartString *to_sm = new CSmartString(parameters["sp_to_file"],delim);
	to_sm->CSplit(to_files);
	delete to_sm;

	// in order to concurrently execute commands
	vector<CShellProcess> shell_cmds;

	int to_file_count = 0;
	list<string>::iterator from_file = from_files.begin();
	while(from_file!=from_files.end()) {
		string ffile = *from_file;
		string tfile = "";

		if(to_files.size()>0) {
			list<string>::iterator to_file = to_files.begin();
			for(int i=0;to_file!=to_files.end();i++, to_file++) {
				if(to_file_count==i) {
					tfile = *to_file;
					to_file_count++;
					break;
				}
			}

			if(tfile=="") {
				list<string>::reverse_iterator reverse_to_file = to_files.rbegin();
				tfile = *reverse_to_file;
			}
		}

		// get relative patch
		char command[1024];
		sprintf(command,"rsync %s %s%s/%s %s%s/%s", parameters["sp_rsync_options"].c_str(), parameters["sp_from_host"].c_str(), parameters["sp_from_path"].c_str(), ffile.c_str(), parameters["sp_to_host"].c_str(), parameters["sp_to_path"].c_str(), tfile.c_str());

		// save commands to concurrently use rsync
		string cmd = command;
		shell_cmds.push_back(CShellProcess(cmd));

		from_file++;
	}

	// in order to concurrently execute commands
	vector<CShellProcess>::iterator current = shell_cmds.begin();
	while(current!=shell_cmds.end()) {
		current->start();
		current++;
	}

	// wait until shell commands are finished
	vector<CShellProcess>::iterator finished = shell_cmds.begin();
	while(finished!=shell_cmds.end()) {
		if(finished->waits()<0) {
			cout << "/////////////////////////////////////////////////////" << endl;
			cout << finished->get_pid() << " was abnormally exited" << endl;
			cout << "/////////////////////////////////////////////////////" << endl;
			exit(-1);
		}
		finished++;
	}

	//inspectParameters(name, parameters);

	//return results;
}
//----------------------------------------------------------------------
void CIndexInterpreter::fetchByRSYNC(string &name, map<string,string> &parameters)
{
	string home = "";

	if(parameters["sp_home"]=="") {
		cout << "[sp_fetch_by_rsync] error : @sp_home must be set" << endl;
		//return results;
	}

	home = parameters["sp_home"];

	if(parameters["sp_from_host"]==""||parameters["sp_from_path"]==""||parameters["sp_from_file"]=="") {
		cout << "[sp_fetch_by_rsync] error : @sp_from_xxxx must be set" << endl;
		//return results;
	}

	parameters["sp_from_host"] += "::R";

	// unconditionally setting
	parameters["sp_to_host"] = "";

	if(parameters["sp_to_path"]=="") {
		string docs_today = parameters["sp_docs"] + "/" + parameters["sp_today"] + "/";
		generateDirectory(home,docs_today);

		parameters["sp_to_path"] = docs_today.c_str();
	}
	else {
		string empty = "";
		generateDirectory(empty, parameters["sp_to_path"]);
	}


	deliverByRSYNC(name, parameters);

	//inspectParameters(name, parameters);

	//return results;
}
//----------------------------------------------------------------------
void CIndexInterpreter::dispatchByRSYNC(string &name, map<string,string> &parameters)
{
	string home = "";

	if(parameters["sp_home"]=="") {
		cout << "[sp_dispatch_by_rsync] error : @sp_home must be set" << endl;
		//return results;
	}

	home = parameters["sp_home"];

	if(parameters["sp_to_host"]=="") {
		cout << "[sp_dispatch_by_rsync] error : @sp_to_xxxx must be set" << endl;
		//return results;
	}

	string target_host = parameters["sp_to_host"];

	parameters["sp_to_host"] += "::R";

	string docs_today = parameters["sp_docs"] + "/" + parameters["sp_today"] + "/";
	string idx_today = parameters["sp_idx"] + "/" + parameters["sp_today"] + "/";
	string conf_path = parameters["sp_conf"] + "/";
	string log_path = parameters["sp_log"] + "/";
	string bin_path = parameters["sp_bin"] + "/";

	// get relative path including today
	string dpath;
	string ipath;
	string cpath;
	string lpath;
	string bpath;

	if(parameters["sp_to_path"]=="") parameters["sp_to_path"] = home;
	if(parameters["sp_to_file"]=="") {
		int start = docs_today.find(home);
		int len = home.size();
		if(start>=0&&start<len) dpath = docs_today.substr(start+len);

		start = idx_today.find(home);
		if(start>=0&&start<len) ipath = idx_today.substr(start+len);

		start = conf_path.find(home);
		if(start>=0&&start<len) cpath = conf_path.substr(start+len);

		start = log_path.find(home);
		if(start>=0&&start<len) lpath = log_path.substr(start+len);

		start = bin_path.find(home);
		if(start>=0&&start<len) bpath = bin_path.substr(start+len);

		//-----------------------------
		// generate remote directories
		//-----------------------------
		char remote_cmd[1024];
		sprintf(remote_cmd,"ssh %s \"mkdir -p %s%s\"", target_host.c_str(), parameters["sp_to_path"].c_str(), dpath.c_str());
		cout << "[execute] : " << remote_cmd << endl;
		system(remote_cmd);
		sprintf(remote_cmd,"ssh %s \"mkdir -p %s%s\"", target_host.c_str(), parameters["sp_to_path"].c_str(), ipath.c_str());
		cout << "[execute] : " << remote_cmd << endl;
		system(remote_cmd);
		sprintf(remote_cmd,"ssh %s \"mkdir -p %s%s\"", target_host.c_str(), parameters["sp_to_path"].c_str(), cpath.c_str());
		cout << "[execute] : " << remote_cmd << endl;
		system(remote_cmd);
		sprintf(remote_cmd,"ssh %s \"mkdir -p %s%s\"", target_host.c_str(), parameters["sp_to_path"].c_str(), lpath.c_str());
		cout << "[execute] : " << remote_cmd << endl;
		system(remote_cmd);
		sprintf(remote_cmd,"ssh %s \"mkdir -p %s%s\"", target_host.c_str(), parameters["sp_to_path"].c_str(), bpath.c_str());
		cout << "[execute] : " << remote_cmd << endl;
		system(remote_cmd);
		//-----------------------------

		string target_paths = dpath + " ";
		target_paths += ipath + " ";
		target_paths += cpath + " ";
		target_paths += lpath + " ";
		target_paths += bpath + " ";
		parameters["sp_to_file"] = target_paths;
	}

	// unconditionally setting
	parameters["sp_from_host"] = "";

	if(parameters["sp_from_path"]=="") parameters["sp_from_path"] = home;
	if(parameters["sp_from_file"]=="") {
		int start = docs_today.find(home);
		int len = home.size();
		if(start>=0&&start<len) dpath = docs_today.substr(start+len);
		
		start = idx_today.find(home);
		if(start>=0&&start<len) ipath = idx_today.substr(start+len);

		start = conf_path.find(home);
		if(start>=0&&start<len) cpath = conf_path.substr(start+len);

		start = log_path.find(home);
		if(start>=0&&start<len) lpath = log_path.substr(start+len);

		start = bin_path.find(home);
		if(start>=0&&start<len) bpath = bin_path.substr(start+len);

		string target_paths = dpath + " ";
		target_paths += ipath + " ";
		target_paths += cpath + " ";
		target_paths += lpath + " ";
		target_paths += bpath + " ";

		parameters["sp_from_file"] = target_paths;
	}

	deliverByRSYNC(name, parameters);

	//inspectParameters(name, parameters);

	//return results;
}
//----------------------------------------------------------------------
void CIndexInterpreter::makeupCommonIndexBlock(string &name, map<string,string> &parameters)
{
	string home = "";

	if(parameters["sp_home"]=="") {
		cout << "[sp_makeup_common_index_block] error : @sp_home must be set" << endl;
		//return results;
	}

	home = parameters["sp_home"];

	if(parameters["uspec_file_to_index"]==""||parameters["uspec_index_group_name"]=="") {
		cout << "[sp_makeup_common_index_block] error : @uspec_xxx related variables must be set" << endl;
		//return results;
	}

	cout << "//////// index : " << parameters["uspec_index_group_name"] << " ////////" << endl;

	char tmp_path[1024];
	sprintf(tmp_path,"%s/%s/%s/",parameters["sp_idx"].c_str(), parameters["sp_today"].c_str(), parameters["uspec_index_group_name"].c_str());

	string index_home = tmp_path;
	string data_coll = index_home + "data_coll";
	string indexed_coll = index_home + "indexed_coll";

	generateDirectory(home, data_coll);
	generateDirectory(home, indexed_coll);

	string docs_today = parameters["sp_docs"] + "/" + parameters["sp_today"] + "/";
	string file_to_index = docs_today + parameters["uspec_file_to_index"];

	/////////////////////////// iinfo ///////////////////////////////////
	string index_info_file_name = indexed_coll + "/index.inf";
	ofstream iinfo_file(index_info_file_name.c_str(), ios::app|ios::out);
	//////////////////////////////////////////////////////////////////////

	string uspec_file_name = indexed_coll + "/SPEC.USR";
	ofstream uspec_file(uspec_file_name.c_str(), ios::app|ios::out);

	// make a user spec file
	uspec_file << "# -------------------------------------------" << endl;
	uspec_file << "# user spec automatically generated by XTask" << endl;
	uspec_file << "# -------------------------------------------" << endl;

	char cmd[1024];
	sprintf(cmd, "ls -alsdF %s | awk '{ n = $10; } END { print $10 }' | xargs", file_to_index.c_str());
	string command = cmd;
	string file_to_inspect = executeShellCommand(command);

	CDocument *doc = new CDocument(file_to_inspect);
	if(doc->hasMoreDocuments()) {
		map<string,string> sections = doc->getNextDocument();
		map<int,string> tags = doc->getTags();

		//--------- num_of_all_sections ----------//
		uspec_file << "num_of_all_sections\t" << tags.size() << endl;
		uspec_file << endl;

		/////////////////////////// iinfo ///////////////////////////////////
		iinfo_file << "iinfo_begin_tag = " << tags[0] << endl;
		/////////////////////////////////////////////////////////////////////

		//---------- section.#.string -----------//
		for(int i=0;i<tags.size();i++) {
			uspec_file << "section." << i+1 << ".string\t" << "\"" << tags[i] << "\"" << endl;
		}
		uspec_file << endl;
	}
	delete doc;


	//---------- collection_dir -----------//
	uspec_file << "collection_dir\t" << data_coll << endl;
	uspec_file << endl;

	//---------------------------------------------------------//
	// start_/end_pivotterm, language, data_coll_dir,
	// num_of_charflts/_intflts/_floatflts/,
	// vintflt./strflt.string, totalcoll_doccnt, 
	// num_of_displaysections
	//---------------------------------------------------------//
	vector<string> opt_params;
	opt_params.push_back("uspec_start_pivotterm");
	opt_params.push_back("uspec_end_pivotterm");
	opt_params.push_back("uspec_language");
	opt_params.push_back("uspec_data_coll_dir");
	opt_params.push_back("uspec_num_of_charflts");
	opt_params.push_back("uspec_num_of_intflts");
	opt_params.push_back("uspec_num_of_floatflts");
	opt_params.push_back("uspec_vintflt.string");
	opt_params.push_back("uspec_strflt.string");
	opt_params.push_back("uspec_totalcoll_doccnt");
	opt_params.push_back("uspec_num_of_displaysections");

	string uspec_pattern = "uspec_";
	string pattern = "uspec_num_of_";
	vector<string>::iterator opt = opt_params.begin();
	while(opt!=opt_params.end()) {
		string opt_param = *opt;
		string opt_val = parameters[opt_param];

		/////////////////////////// iinfo ///////////////////////////////////
		if(opt_param=="uspec_num_of_charflts"&&opt_val!="") iinfo_file << "iinfo_filter = charflt" << endl;
		else if(opt_param=="uspec_num_of_intflts"&&opt_val!="") iinfo_file << "iinfo_filter = intflt" << endl;
		else if(opt_param=="uspec_num_of_floatflts"&&opt_val!="") iinfo_file << "iinfo_filter = floatflt" << endl;
		else if(opt_param=="uspec_vintflt.string"&&opt_val!="") iinfo_file << "iinfo_filter = vintflt" << endl;
		else if(opt_param=="uspec_strflt.string"&&opt_val!="") iinfo_file << "iinfo_filter = strflt" << endl;
		/////////////////////////////////////////////////////////////////////

		if(opt_val!="") {
			uspec_file << opt_param.substr(uspec_pattern.length()) << "\t" << opt_val << endl;

			if(opt_param.find(pattern)==0) {
				string extra_pattern = uspec_pattern + opt_param.substr(pattern.length());
				// in the case of "num_of_xxxflts", find "xxxflt.#.string"
				if(extra_pattern.rfind("flts")==extra_pattern.length()-4) extra_pattern = extra_pattern.substr(0,extra_pattern.length()-1);
				// in the case of "num_of_displaysections", find "displaysec.#.string/.type"
				else if(extra_pattern.rfind("sections")==extra_pattern.length()-8) extra_pattern = extra_pattern.substr(0,extra_pattern.length()-5);

				map<string,string>::iterator param = parameters.begin();
				while(param!=parameters.end()) {
					string attr = param->first;
					string val = param->second;

					if(attr.find(extra_pattern)==0) {
						uspec_file << attr.substr(uspec_pattern.length()) << "\t" << val << endl;
					}

					param++;
				}
			}

			uspec_file << endl;
		}

		opt++;
	}

	//---------  num_of_indexes ----------//
	int task_found = 0;
	int index_count = 0;

	string imap_file_name = parameters["sp_tdl_file_name"];
	CInfoMap *im = new CInfoMap((char *)imap_file_name.c_str());
	while(im->hasMoreTasks()) {
		CTask task = im->getNextTask();
		//if(task.getName()!="index") continue;

		string file_to_index = task.getParameter("@uspec_file_to_index");
		string index_group_name = task.getParameter("@uspec_index_group_name");

		task_found = 0;

		if(file_to_index==parameters["uspec_file_to_index"]&&index_group_name==parameters["uspec_index_group_name"]) {
			index_count = 0;

			while(task.hasMoreWorks()) {
				CWork work = task.getNextWork();

				string type = work.getType();
				if(type!="local"&&type!="remote") continue;


				if(work.getName()=="@sp_makeup_common_index_block") {
					task_found = 1;
				}

				if(task_found&&work.getName()=="@sp_makeup_index_block") {
					index_count++;
				}
			}

			if(task_found) break;
		}
	}
	delete im;

	if(task_found&&index_count>0) {
		uspec_file << "num_of_indexes\t" << index_count << endl;
		uspec_file << endl;
	}
	else {
		cout << "[sp_makeup_common_index_block] error : num_of_indexes was not extracted, so check the variables : @uspec_file_to_index, @uspec_index_group_name, @uspec_index_name" << endl;
	}

	uspec_file.close();
	cout << "[common_index_block] : created a user spec file : " << uspec_file_name << endl;

	/////////////////////////// iinfo ///////////////////////////////////
	iinfo_file.close();
	/////////////////////////////////////////////////////////////////////

	//inspectParameters(name, parameters);

	//return results;
}
//----------------------------------------------------------------------
void CIndexInterpreter::makeupIndexBlock(string &name, map<string,string> &parameters)
{
    string home = "";

    if(parameters["sp_home"]=="") {
        cout << "[sp_makeup_index_block] error : @sp_home must be set" << endl;
        //return results;
    }

    home = parameters["sp_home"];

    if(parameters["uspec_file_to_index"]==""||parameters["uspec_index_name"]=="") {
        cout << "[sp_makeup_index_block] error : @uspec_xxx related variables must be set" << endl;
        //return resutls;
    }

	cout << "//////// index : " << parameters["uspec_index_group_name"] << " ////////" << endl;

    char tmp_path[1024];
    sprintf(tmp_path,"%s/%s/%s/",parameters["sp_idx"].c_str(), parameters["sp_today"].c_str(), parameters["uspec_index_group_name"].c_str());

    string index_home = tmp_path;
    string data_coll = index_home + "data_coll";
    string indexed_coll = index_home + "indexed_coll";

    generateDirectory(home, data_coll);
    generateDirectory(home, indexed_coll);

    string docs_today = parameters["sp_docs"] + "/" + parameters["sp_today"] + "/";
    string file_to_index = docs_today + parameters["uspec_file_to_index"];

	/////////////////////////// iinfo ///////////////////////////////////
	string index_info_file_name = indexed_coll + "/index.inf";
	ofstream iinfo_file(index_info_file_name.c_str(), ios::app|ios::out);
	//////////////////////////////////////////////////////////////////////	

    string uspec_file_name = indexed_coll + "/SPEC.USR";
    ofstream uspec_file(uspec_file_name.c_str(), ios::app|ios::out);

	/////////////////////////// iinfo ///////////////////////////////////
	iinfo_file << "iinfo_index_name = " << parameters["uspec_index_name"] << endl;
	/////////////////////////////////////////////////////////////////////

    // make a user spec file
	//---------- <INDEX ...> ----------//
	uspec_file << "<INDEX " << parameters["uspec_index_name"] << ">" << endl;

	//---------- index_volume_dir -----------//
	uspec_file << "index_volume_dir\t" << indexed_coll << "/" << parameters["uspec_index_name"] << endl;
	uspec_file << endl;

	//--------- num_of_extidx_sections ----------//
	int task_found = 0;
	int index_found = 0;
	int section_count = 0;

	string imap_file_name = parameters["sp_tdl_file_name"];
	CInfoMap *im = new CInfoMap((char *)imap_file_name.c_str());
	while(im->hasMoreTasks()) {
		CTask task = im->getNextTask();
		//if(task.getName()!="index") continue;

		string file_to_index = task.getParameter("@uspec_file_to_index");
		string index_group_name = task.getParameter("@uspec_index_group_name");

		task_found = 0;
		index_found = 0;

		if(file_to_index==parameters["uspec_file_to_index"]&&index_group_name==parameters["uspec_index_group_name"]) {
			while(task.hasMoreWorks()) {
				CWork work = task.getNextWork();

				string type = work.getType();
				if(type!="local"&&type!="remote") continue;

				if(work.getName()=="@sp_makeup_index_block") {
					task_found = 1;
				}

				index_found = 0;

				if(task_found&&work.getName()=="@sp_makeup_index_block") {
					string index_name = work.getParameter("@uspec_index_name");

					if(index_name==parameters["uspec_index_name"]) {
						index_found = 1;

						section_count = 0;
						int max_num_of_section_to_index = atoi(parameters["uspec_max_num_of_section_to_index"].c_str());
						for(int i=1;i<max_num_of_section_to_index;i++) {
							char pttrn[1024];
							sprintf(pttrn, "uspec_section.%d.string", i);
							string section_pattern = pttrn;

							string found_pattern = parameters[section_pattern];

							if(found_pattern!="") section_count++;
							else break;
						}
					}
				}

				if(index_found) break;
			}

			if(task_found) break;
		}
	}
	delete im;

	if(task_found&&index_found&&section_count>0) {
		uspec_file << "num_of_extidx_sections\t" << section_count << endl;
		uspec_file << endl;
	}
	else {
		cout << "[sp_makeup_index_block] error : num_of_extidx_sections was not extracted, so check the variables : @uspec_file_to_index, @uspec_index_group_name, @uspec_index_name" << endl;
	}

	//---------- section.#.string/.indexer/.importance/.parameter ---------//
	char pttrn[1024];
	string section_param;
	string uspec_pattern= "uspec_";
	for(int i=1;i<=section_count;i++) {
		sprintf(pttrn, "section.%d.string", i);
		section_param = uspec_pattern + pttrn;
		uspec_file << pttrn << "\t" << parameters[section_param] << endl;

		sprintf(pttrn, "section.%d.indexer", i);
		section_param = uspec_pattern +pttrn;
		if(parameters[section_param]!="") uspec_file << pttrn << "\t" << parameters[section_param] << endl;
		else uspec_file << pttrn << "\t" << parameters["uspec_section.indexer"] << endl;

		sprintf(pttrn, "section.%d.importance", i);
		section_param = uspec_pattern + pttrn;
		uspec_file << pttrn << "\t" << parameters[section_param] << endl;

		sprintf(pttrn, "section.%d.parameter", i);
		section_param = uspec_pattern +pttrn;
		if(parameters[section_param]!="") uspec_file << pttrn << "\t" << parameters[section_param] << endl;
		else uspec_file << pttrn << "\t" << parameters["uspec_section.parameter"] << endl;

		uspec_file << endl;
	}

	//----------- section.score.string/.importance/.parameter -----------//
	vector<string> score_params;
	score_params.push_back("uspec_section.score.string");
	score_params.push_back("uspec_section.score.importance");
	score_params.push_back("uspec_section.score.parameter");

	vector<string>::iterator score = score_params.begin();
	while(score!=score_params.end()) {
		string score_param = *score;
		string score_val = parameters[score_param];

		if(score_val!="") {
			uspec_file << score_param.substr(uspec_pattern.length()) << "\t" << score_val << endl;
		}

		score++;
	}
	uspec_file << endl;

	//----------- num_of_scores, score.# -------------//
	int num_of_scores = atoi(parameters["uspec_num_of_scores"].c_str());
	if(num_of_scores!=0) {
		uspec_file << "num_of_scores\t" << num_of_scores << endl;

		string score_param;
		for(int i=1;i<=num_of_scores;i++) {
			char pttrn[1024];
			sprintf(pttrn, "score.%d", i);
			score_param = uspec_pattern + pttrn;
			uspec_file << pttrn << "\t" << parameters[score_param] << endl;
		}
	}
	uspec_file << endl;

	//----------- query.indexer ------------//
	uspec_file << "query.indexer\t" << parameters["uspec_query.indexer"] << endl;
	uspec_file << endl;

	//---------------------------------------------------------------------//
	// wordspam, wordspam_body_threshold/_rate/_spamcount
	// wordspam.#.threshold/.rate/.spamcount
	// within_default_distance, near_default_distance
	// rac, rac_block_size/_df_threshold/_proximity
	// lex_block_size, df_threshold, exclude_gtf1_term
	// doclength_threshold, doclength_norm_factor
	// sim_norm_mode, truncation_type, proximity_data, dvec_data
	// case_sensitive, save_topweightdocs, topweightdoccnt, topweighttermcnt
	//---------------------------------------------------------------------//
	vector<string> opt_params;
	opt_params.push_back("uspec_wordspam");
	opt_params.push_back("uspec_wordspam_body_threshold");
	opt_params.push_back("uspec_wordspam_body_rate");
	opt_params.push_back("uspec_wordspam_body_spamcount");
	opt_params.push_back("uspec_wordspam.#.threshold");
	opt_params.push_back("uspec_wordspam.#.rate");
	opt_params.push_back("uspec_wordspam.#.spamcount");
	opt_params.push_back("uspec_truncation_type");
	opt_params.push_back("uspec_proximity_data");
	opt_params.push_back("uspec_within_default_distance");
	opt_params.push_back("uspec_near_default_distance");
	opt_params.push_back("uspec_case_sensitive");
	opt_params.push_back("uspec_dvec_data");
	opt_params.push_back("uspec_save_topweightdocs");
	opt_params.push_back("uspec_topweightdoccnt");
	opt_params.push_back("uspec_topweighttermcnt");
	opt_params.push_back("uspec_rac");
	opt_params.push_back("uspec_rac_block_size");
	opt_params.push_back("uspec_rac_df_threshold");
	opt_params.push_back("uspec_rac_proximity");
	opt_params.push_back("uspec_lex_block_size");
	opt_params.push_back("uspec_df_threshold");
	opt_params.push_back("uspec_exclude_gtf1_term");
	opt_params.push_back("uspec_doclength_threshold");
	opt_params.push_back("uspec_doclength_norm_factor");
	opt_params.push_back("uspec_sim_norm_mode");

	vector<string>::iterator opt = opt_params.begin();
	while(opt!=opt_params.end()) {
		string opt_param = *opt;
		string opt_val = parameters[opt_param];

		if(opt_val!="") {
			uspec_file << opt_param.substr(uspec_pattern.length()) << "\t" << opt_val << endl;;
		}

		opt++;
	}

	//---------- </INDEX> ----------//
	uspec_file << "</INDEX>" << endl;
	uspec_file << endl;

	uspec_file.close();
	cout << "[index_block] : created a user spec file : " << uspec_file_name << endl;

	/////////////////////////// iinfo ///////////////////////////////////
	iinfo_file.close();
	/////////////////////////////////////////////////////////////////////

	//inspectParameters(name, parameters);

	//return results;
}
//----------------------------------------------------------------------
void CIndexInterpreter::index(string &name, map<string,string> &parameters)
{
    if(parameters["sp_home"]=="") {
        cout << "[sp_index] error : @sp_home must be set" << endl;
        //return results;
    }

	// get index-related infomation from index.inf
	/////////////////////////// iinfo ///////////////////////////////////
	char line[MAX_LINE_LENGTH];
	char tmp_path[1024];
	sprintf(tmp_path,"%s/%s/%s/",parameters["sp_idx"].c_str(), parameters["sp_today"].c_str(), parameters["uspec_index_group_name"].c_str());
	string index_home = tmp_path;
	string indexed_coll = index_home + "indexed_coll";
	string iinfo_file_name = indexed_coll + "/index.inf";
	ifstream iinfo_file(iinfo_file_name.c_str());
	while(iinfo_file) {
		iinfo_file.getline(line,MAX_LINE_LENGTH);

		string av_line = line;

		string delim = " \t=";
		list<string> attr_val;
		CSmartString *index_av = new CSmartString(av_line, delim);
		index_av->CSplit(attr_val);
		delete index_av;

		string attr = "";
		string val = "";
		list<string>::iterator av = attr_val.begin();

		if(av!=attr_val.end()) attr = *av;
		av++;
		if(av!=attr_val.end()) val = *av;

		if(attr=="iinfo_index_name") parameters[attr] += (" " + val);
		else if(attr=="iinfo_filter") parameters[attr] += (" " + val);
		else if(attr!="") parameters[attr] = val;
	}
	iinfo_file.close();
	
	remove(iinfo_file_name.c_str());
	//////////////////////////////////////////////////////////////////////

	// process sigmadloc
	executeSigmaDocumentLocation(parameters);

	// process sigmaindex
	executeSigmaIndex(parameters);

	// process sigmafilter
	executeSigmaFilter(parameters);


	//inspectParameters(name, parameters);

	//return results;
}
//----------------------------------------------------------------------
void CIndexInterpreter::makeupNXP(string &name, map<string,string> &parameters) 
{
	string home = "";

	if(parameters["sp_home"]=="") {
		cout << "[sp_makeup_nxp] error : @sp_home must be set" << endl;
		//return results;
	}

	home = parameters["sp_home"];

	if(parameters["nxpspec_server_name"]==""||parameters["nxpspec_port_num"]==""||parameters["nxpspec_index_group_name"]=="") {
		cout << "[sp_makeup_nxp] error : @nxpspec_server_name, @nxpspec_port_num, or @nxpspec_index_group_name must be set" << endl;
		//return results;
	}

	char tmp_path[1024];
	sprintf(tmp_path,"%s/%s/%s/",parameters["sp_idx"].c_str(), parameters["sp_today"].c_str(), parameters["nxpspec_index_group_name"].c_str());

	string index_home = tmp_path;
	string data_coll = index_home + "data_coll";
	string indexed_coll = index_home + "indexed_coll";

	string nxpspec_file_name = parameters["sp_conf"] + "/" + parameters["nxpspec_server_name"] + "_nxp.conf";

	// remove previous _nxp.conf file
	remove(nxpspec_file_name.c_str());

	// create new _nxp.conf file
	ofstream nxpspec_file(nxpspec_file_name.c_str(), ios::app|ios::out);

	// make a nxpserver/nxpbroker spec file
	nxpspec_file << "# ------------------------------------------------------" << endl;
	nxpspec_file << "# nxpserver/broker spec automatically generated by XTask" << endl;
	nxpspec_file << "# ------------------------------------------------------" << endl;

	//----------- ServerType/Name, NickName, PortNum ------------//
	nxpspec_file << "ServerType\t\t" << parameters["nxpspec_server_type"] << endl;
	nxpspec_file << "ServerName\t\t" << parameters["nxpspec_server_name"] << endl;
	if(parameters["nxpspec_nick_name"]=="") nxpspec_file << "NickName\t\t" << parameters["nxpspec_server_name"] << endl;
	else nxpspec_file << "NickName\t\t" << parameters["nxpspec_nick_name"] << endl;
	nxpspec_file << "PortNum\t\t" << parameters["nxpspec_port_num"] << endl;
	nxpspec_file << endl;

	//------------------------------------------------------//
	// MaxServers, StartServers
	// MinSpareServers, MaxSpareServers, MaxRequestPerChild
	// MaxClients
	// BackLog, TraceLevel
	//-----------------------------------------------------//
	string nxpspec_pattern = "_nxpspec_";
	vector<string> opt_common_params;
	opt_common_params.push_back("MaxServers_nxpspec_max_servers");
	opt_common_params.push_back("StartServers_nxpspec_start_servers");
	opt_common_params.push_back("MinSpareServers_nxpspec_min_spare_servers");
	opt_common_params.push_back("MaxSpareServers_nxpspec_max_spare_servers");
	opt_common_params.push_back("MaxRequestPerChild_nxpspec_max_request_per_child");
	opt_common_params.push_back("MaxClients_nxpspec_max_clients");
	opt_common_params.push_back("BackLog_nxpspec_back_log");
	opt_common_params.push_back("TraceLevel_nxpspec_trace_level");

	vector<string>::iterator opt_common = opt_common_params.begin();
	while(opt_common!=opt_common_params.end()) {
		string opt_common_param = *opt_common;
		int delim = opt_common_param.find(nxpspec_pattern);
		string opt_common_name = opt_common_param.substr(0,delim);
		string opt_common_val = parameters[opt_common_param.substr(delim+1)];

		if(opt_common_val!="") {
			if(opt_common_name=="MaxClients"||opt_common_name=="BackLog"||opt_common_name=="TraceLevel") nxpspec_file << endl;

			nxpspec_file << opt_common_name << "\t\t" << opt_common_val << endl;
		}

		opt_common++;
	}
	nxpspec_file << endl;


	/*
	//------------- srcache --------------//
	nxpspec_file << endl;


	//------------- monitorserver/interval -------------//
	nxpspec_file << endl;
	*/


	//------------- UsrSpecFile_Indexes --------------//
	string coll_defname = parameters["nxpspec_index_group_name"];
	string uspec_file = indexed_coll + "/SPEC.USR";
	string index_names = "";

	if(parameters["nxpspec_index_name"]!="") {
		string delim = ",:; \t";
		list<string> index_files;
		CSmartString *sm =  new CSmartString(parameters["nxpspec_index_name"],delim);
		sm->CSplit(index_files);
		delete sm;

		nxpspec_file << "UsrSpecFile_Indexes\t" << coll_defname << "\t" << uspec_file << "\t";

		list<string>::iterator ifile = index_files.begin();
		while(ifile!=index_files.end()) {
			nxpspec_file << *ifile << " "; 

			ifile++;
		}

		nxpspec_file << endl;
	}
	else {
		vector<string> dirs;
		int level = 1;
		CFileManager *fileman = new CFileManager();
		fileman->StartWalking((char *)indexed_coll.c_str(), level);
		dirs = fileman->getDirs(dirs);
		delete fileman;

		nxpspec_file << "UsrSpecFile_Indexes\t" << coll_defname << "\t" << uspec_file << "\t";

		for(int i=0;i<dirs.size();i++) {
			string fullpath = dirs[i];
			string index_file = fullpath.substr(fullpath.rfind("/")+1);

			nxpspec_file << index_file << " ";
		}

		nxpspec_file << endl;
	}

	nxpspec_file << endl;


	//------------------------------------------------//
	// ClientSpec
	// LinkedServer, ConnTimeOut
	// MergeType/Ratio
	// TopkWantedNum
	//------------------------------------------------//
	vector<string> opt_broker_params;
	opt_broker_params.push_back("ClientSpec_nxpspec_client_spec");
	opt_broker_params.push_back("LinkedServer_nxpspec_linked_server");
	opt_broker_params.push_back("ConnTimeOut_nxpspec_conn_time_out");
	opt_broker_params.push_back("MergeType_nxpspec_merge_type");
	opt_broker_params.push_back("MergeRatio_nxpspec_merge_ratio");
	opt_broker_params.push_back("TopkWantedNum_nxpspec_top_k_wanted_num");

	vector<string>::iterator opt_broker = opt_broker_params.begin();
	while(opt_broker!=opt_broker_params.end()) {
		string opt_broker_param = *opt_broker;
		int delim = opt_broker_param.find(nxpspec_pattern);
		string opt_broker_name = opt_broker_param.substr(0,delim);
		string opt_broker_val = parameters[opt_broker_param.substr(delim+1)];

		if(opt_broker_val!="") {
			nxpspec_file << opt_broker_name << "\t\t" << opt_broker_val << endl;
		}

		opt_broker++;
	}


	nxpspec_file.close();
	cout << "[nxpserver/broker] : created a nxp spec file : " << nxpspec_file_name << endl;
	        
	//inspectParameters(name, parameters);

	//return results;
}
//----------------------------------------------------------------------
void CIndexInterpreter::nxp(string &name, map<string,string> &parameters)
{
	if(parameters["sp_home"]=="") {
		inspectParameters(name, parameters);

		cout << "[sp_nxp] error : @sp_home must be set" << endl;
		//return results;
	}

	if(parameters["nxp_host"]=="") {
		inspectParameters(name, parameters);

		cout << "[sp_nxp] error : @nxp_host must be set" << endl;
		//return results;
	}

	string delim = ",:; \t";
	list<string> operations;
	CSmartString *sm = new CSmartString(parameters["nxp_operations"], delim);
	sm->CSplit(operations);
	delete sm;

	list<string>::iterator op = operations.begin();
	while(op!=operations.end()) {
		string operation = *op;

		if(operation=="stop") stopNXP(parameters);
		else if(operation=="start") startNXP(parameters);
		else if(operation=="restart") {
			stopNXP(parameters);
			startNXP(parameters);
		}

		op++;
	}


	//inspectParameters(name, parameters);

	//return results;
}
//----------------------------------------------------------------------
